import os, sys, time, json
import re
import requests
from lxml import etree
from selenium import webdriver

pwd = os.getcwd()
sys.path.append(pwd)
from proxy import ValidIp

# base_url = 'https://car.autohome.com.cn/config/series/{}.html'
base_url = 'https://car.autohome.com.cn/config/spec/{}.html'
html_dir = '/home/ubuntu/workspace/spider_data/qichezhijia/html2/'
newhtml_dir = '/home/ubuntu/workspace/spider_data/qichezhijia/newhtml2/'
json_dir = '/home/ubuntu/workspace/spider_data/qichezhijia/json2/'
content_dir = '/home/ubuntu/workspace/spider_data/qichezhijia/content2/'
newjson_dir = '/home/ubuntu/workspace/spider_data/qichezhijia/newjson2/'
zaishou_filepath = 'youzaishou1.txt'
host = 'http://192.168.50.149:4010'
'//*[@id="mybody"]/div[5]/div[5]/table[1]/tbody/tr[76]/td[2]/div'
''

# 向接口发送数据
def post_zaishou_data(data):
    r = requests.post(host + '/api/v1/qichezhijia/zaishou', data=data)
    return(r.text)

def post_tingshou_data(data):
    r = requests.post(host + '/api/v1/qichezhijia/tingshou', data=data)
    return(r.text)

# 1. 下载html
def download_html(filepath=zaishou_filepath):
    with open(zaishou_filepath, 'r') as f:
        number_content = f.read()
    list_number = number_content.split(',')
    proxies = ValidIp()
    for number in list_number:
        url = base_url.format(number)
        response = requests.get(url, proxies=proxies)
        response.encoding='utf8'
        print(number)
        with open(html_dir+str(number),'a',encoding='utf-8') as f:
            f.write(response.text)

# 2. 解析出每个车型的关键js拼装成一个html
def create_newhtml(html_dir=html_dir):
    html_files = os.listdir(html_dir)
    for html_file in html_files:
        print("fileName=="+html_file)
        with open(html_dir+html_file,'r',encoding="utf-8") as f:
            html_content = f.read()
        # 解析数据的json
        alljs = ('''var rules = '2';
                    var document = {};
                    function getRules(){return rules}
                    document.createElement = function() {
                        return {
                            sheet: {
                                insertRule: function(rule, i) {
                                    if (rules.length == 0) {
                                        rules = rule;
                                    } else {
                                        rules = rules + '#' + rule;
                                    }
                                }
                            }
                        }
                    };
                    document.querySelectorAll = function() {
                       return {};
                    };
                    document.head = {};
                    document.head.appendChild = function() {};
                    var window = {};
                    window.decodeURIComponent = decodeURIComponent;''')
        try:
            js = re.findall('(\(function\([a-zA-Z]{2}.*?_\).*?\(document\);)', html_content)
            for item in js:
                alljs = alljs + item
        except Exception as e:
            print('makejs function exception:%s',e)

        newHtml = "<html><meta http-equiv='Content-Type' content='text/html; charset=utf-8' /><head></head><body>    <script type='text/javascript'>"
        alljs = newHtml + alljs+" document.write(rules)</script></body></html>"
        with open(newhtml_dir+html_file+'.html',"a",encoding="utf-8") as f:
            f.write(alljs)

# 在js代码里面获取json数据
def join_json_data(list_pattern, html_content):
    # 将几个json数据连接到json_data中
    json_data = ''
    for pattern in list_pattern:
        result = re.search(pattern, html_content)
        if result!= None:
            json_data += result.group(0)
    return json_data

# 3. 解析出每个车型的数据json，保存到本地。
def create_json(html_dir=html_dir):
    html_files = os.listdir(html_dir)
    for html_file in html_files:
        print("fileName=="+html_file)
        with open(html_dir+html_file,'r',encoding="utf-8") as f:
            html_content = f.read()
        # config包括：基本参数、车身、发动机、电动机、变速箱、底盘转向、车轮制动
        # option包括：主/被动安全装备、辅助/操控配置、外部/防盗配置、内部配置、座椅配置、多媒体配置、灯光配置、玻璃/后视镜、空调/冰箱
        # color包括：选装包(车身颜色)
        # innerColor包括：选装包(内饰颜色)
        list_pattern = ['var config = (.*?){1,};', 'var option = (.*?)};', 'var color = (.*?);', 'var innerColor =(.*?);']
        json_data = join_json_data(list_pattern, html_content)
        # print(json_data)
        with open(json_dir+html_file,"a",encoding="utf-8") as f:
            f.write(json_data)

# 4. 浏览器执行生成的newhtml文件，抓取执行结果，保存到本地
def create_content(newhtml_dir=newhtml_dir):
    newhtml_files = os.listdir(newhtml_dir)
    browser = webdriver.Chrome()
    for newhtml in newhtml_files:
        browser.get('file://' + newhtml_dir+newhtml)
        body = browser.find_element_by_tag_name('body')
        print(body.text)
        with open(content_dir+newhtml,"a",encoding="utf-8") as f:
            f.write(body.text)
    browser.close()

# 5. 匹配样式文件与json数据文件，生成正常的数据文件。
def create_newjson(json_dir=json_dir):
    list_json = os.listdir(json_dir)
    for json_file in list_json:
        print(json_file)
        # 读取json数据文件
        with open(json_dir+json_file,'r',encoding="utf-8") as f:
            json_content = f.read()
        # 读取样式文件
        content_path = content_dir+json_file+'.html'
        # print(spansPath)
        with open(content_path,"r",encoding="utf-8") as f:
            content = f.read()
        print(content)
        # 获取所有span标签
        # span_all = re.findall("<span(.*?)></span>",json_content)
        classname_all = re.findall("<span class='(.*?)'></span>", json_content)
        for class_name in classname_all:
            # print("匹配到的span=>>"+span)
            # 获取class属性值
            # class_name = re.search("'(.*?)'", span).group(1)
            print("匹配到的class==>"+class_name)
            span_pattern = str(class_name)+"::before { content:\"(.*?)\" }"
            # 匹配样式值
            span_result = re.search(span_pattern, content).group(1)
            if span_result != None:
                if class_name != None:
                    print("匹配到的样式值="+span_result)
                    json_content = json_content.replace("<span class='"+class_name+"'></span>", span_result)
        # print(json_content)
        with open(newjson_dir+json_file,"a",encoding="utf-8") as f:
            f.write(json_content)

# 6. 读取数据文件，生成excel
def save_data(newjson_dir=newjson_dir):
    files = os.listdir(newjson_dir)
    for file in files:
        print("fileName=="+file)
        with open(newjson_dir+file,'r',encoding="utf-8") as f:
            newjson_content = f.read()
        # 解析基本参数配置参数，颜色三种参数，其他参数
        config_pattern = "var config = (.*?);"
        option_pattern = "var option = (.*?);var"
        # bag_pattern = "var bag = (.*?);"
        color_pattern = 'var color = (.*?);'
        innercolor_pattern = 'var innerColor =(.*?);'

        # 获取json字符串
        str_config = re.search(config_pattern, newjson_content).group(1)
        try :
            str_option = re.search(option_pattern, newjson_content).group(1)
        except Exception:
            str_option = ''
        # str_bag = re.search(bag_pattern, newjson_content).group(1)
        str_color = re.search(color_pattern, newjson_content).group(1)
        str_innercolor = re.search(innercolor_pattern, newjson_content).group(1)
        # 把json字符串转为json对象
        config_json = json.loads(str_config)
        option_json = json.loads(str_option)
        # bag_json = json.loads(str_bag)
        color_json = json.loads(str_color)
        innercolor_json = json.loads(str_innercolor)
        # 打印
        # print(config_json)
        # print(option_json)
        # print(color_json)
        # print(innercolor_json)
        # data = {
        #     'jibencanshu': config_json,
        #     'peizhicanshu': option_json,
        #     'waiguanyanse': color_json,
        #     'neishiyanse': innercolor_json,
        # }
        data = {
            'jibencanshu': str_config,
            'peizhicanshu': str_option,
            'waiguanyanse': str_color,
            'neishiyanse': str_innercolor,
        }
        # print(data)
        # post_zaishou_data(data)
        post_tingshou_data(data)


if __name__=='__main__':
    # download_html()
    # create_newhtml()
    # create_json()
    # create_content()
    # create_newjson()
    save_data()